import importlib
import inspect

from utils.database import Kafka
from items import ResponseItem
from utils import logger, get_function_from_path_string
from items import DataBaseItem
from config import settings


@Kafka.consumer.decorator
def spider_result_processor(*args, **kwargs):
    """
    爬虫抓取的数据，使用callback的方法进行解析
    :param args:
    :param kwargs:
    :return:
    """

    def data_success(key, item):
        if isinstance(item, DataBaseItem):
            data_process_result[key].append(item)
        elif item is None:
            pass
        else:
            logger.error("返回的ITEM不是BaseItem类型")

    cache = kwargs.get("cache")
    data_process_result = {}
    for msg in cache:
        resp_item = ResponseItem(**msg.value)

        if not data_process_result.get(resp_item.request_item.data_callback):
            data_process_result[resp_item.request_item.data_callback] = []

        parse_function = get_function_from_path_string(resp_item.request_item.parse_callback)
        if not parse_function or parse_function == settings.undefined:
            logger.warning(f"parse_callback {settings.undefined}!")
            continue

        if inspect.isgeneratorfunction(parse_function):
            for item in parse_function(resp_item):
                data_success(resp_item.request_item.data_callback, item)
        else:
            item = parse_function(resp_item)
            data_success(resp_item.request_item.data_callback, item)

    data_processor(data_process_result)


def data_processor(data_process_result: dict):
    for key, items in data_process_result.items():
        if not items:
            continue
        data_process_function = get_function_from_path_string(key)
        if not data_process_function or data_process_function == settings.undefined:
            logger.warning(f"data_callback {settings.undefined}!")
            continue
        data_process_function(items)
